package com.ylzinfo.ms.fetchdemo;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.junit.Test;

/**
 * @Description:
 * @Product: IntelliJ IDEA
 * @Author Rogchen rogchen128@gmail.com
 * @Created Date: 20-2-12 15:59
 **/
public class 使用htmlutil抓取 {

    @Test
    public void htmlutil() throws Exception {
        String url = "https://ssc.mohrss.gov.cn/main?_api_timestamp=1580909751215&security=VxYGjhhH1yTAm7OgefOywfWQ27nVlhiV8t8a3fQdWgSwh6xcJw1BfT0aE%2BjyXQXuiENOjW%2Bgp89zicigqxzUlTvwh44xeQSPzwNT8poORIbzJFl4cTRv7wjMaZtiH60WAYgzHE7m1R%2Foo6AMnhwTYva0yNa4HkRXGsd6FglBINgEHsAsotPDd0d2diVfRBjBYRgOTEQzJVBLcMMIHqMhYA%3D%3D&_api_name=get_token&api_access_key=15dfa422d02d49108741f011b6a5e3f9&_api_signature=iteFyLXHfdIzy6639R%2FmZ%2FsQHfc%3D&return_url=http%3A%2F%2Fwww.ylzinfo.com&_api_version=1.0.0&_api_access_key=15dfa422d02d49108741f011b6a5e3f9";
        System.out.println("----------------------抓取页面时不解析js-----------------");
//        crawlPageWithoutAnalyseJs(url);
        System.out.println("----------------------抓取页面时解析js-------------------");
        crawlPageWithAnalyseJs(url);
    }

    /**
     * 功能描述：抓取页面时不解析页面的js
     * @param url
     * @throws Exception
     */
    public void crawlPageWithoutAnalyseJs(String url) throws Exception{
        //1.创建连接client
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        //2.设置连接的相关选项
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setJavaScriptEnabled(false);
        webClient.getOptions().setTimeout(10000);
        //3.抓取页面
        HtmlPage page = webClient.getPage(url);
        System.out.println(page.asXml());
        //4.关闭模拟窗口
        webClient.close();
    }

    /**
     * 功能描述：抓取页面时并解析页面的js
     * @param url
     * @throws Exception
     */
    public void crawlPageWithAnalyseJs(String url) throws Exception{
        //1.创建连接client
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        //2.设置连接的相关选项
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setJavaScriptEnabled(true);  //需要解析js
        webClient.getOptions().setThrowExceptionOnScriptError(false);  //解析js出错时不抛异常
        webClient.getOptions().setTimeout(10000);  //超时时间  ms
        //3.抓取页面
        HtmlPage page = webClient.getPage(url);
        //4.将页面转成指定格式
        webClient.waitForBackgroundJavaScript(60000);   //等侍js脚本执行完成
        System.out.println(page.asXml());
        //5.关闭模拟的窗口
        webClient.close();
//        url = page.asXml();
//        String fristURL = url.replace("&lt;", "<");
//        String secondURL = fristURL.replace("&gt;", ">");
//        String thtURL = secondURL.replace("<br/>", "");
//        String fourURL = thtURL.replace("\n", "");
//        String lastURL = fourURL.replace("\t", "");
//
//        org.jsoup.nodes.Document doc = Jsoup.parse(lastURL);
//        Elements elements = doc.getElementsByTag("location");
////        for (Iterator it = elements.iterator(); it.hasNext(); ) {
////            Element node = (Element) it.next();
////        }
//        String html = doc.toString();
//        System.out.println("html+json解析xml文件"+html+"多的对应的标签元素+"+doc.getElementsByTag("location"));
    }
}
